package com.home.component;/**
 * @author chenxf
 * @date 2022/4/15 15:13
 */

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.home.annotation.PolicyNewsProcesses;
import com.home.entity.PolicyNewsInfo;
import com.home.entity.PolicySiteConfig;
import com.home.enums.FetchTypeEnum;
import com.home.service.PolicyNewsIface;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import org.springframework.util.ObjectUtils;

import java.net.MalformedURLException;
import java.net.URL;
import java.sql.Timestamp;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;

/**
 * @author chenxf
 * @date 2022/4/15 15:13
 */
@Component
@PolicyNewsProcesses(type = FetchTypeEnum.GOV_SITE)
public class GovNewsProcessor implements PolicyNewsIface {
    DateTimeFormatter formatter= DateTimeFormatter.ofPattern("yyyy-MM-dd");
    String baseUri="";

    @Override
    public List<PolicyNewsInfo> fetchPage(PolicySiteConfig config,String lastUrl) throws Exception {
        initBaseUrl(config.getSiteUrl());
        //System.out.println("site="+config.getSiteUrl());
        List<PolicyNewsInfo> policyNewsInfos=new ArrayList<>(10);
        Document response = Jsoup.connect(config.getSiteUrl())
                .ignoreContentType(true)
                .method(Connection.Method.GET)
                // .proxy("127.0.0.1",8866)
                // .header("Content-Type","application/json;charset=UTF-8")
                //.header("Content-Type","application/json;charset=UTF-8")
                // .header("sec-ch-ua-platform","Windows")
                .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36")
                //.cookies(cookie)
                .execute().parse();
        //System.out.println(response.body());

        Elements elements = response.getElementsByTag("script");
        Elements dtElements;
        String bodyHtml="";

        for (Element e : elements) {
            bodyHtml = e.data().replace("<![CDATA[", "")
                    .replace("]]>", "");
            Document scriptDoc = Jsoup.parse(bodyHtml);
            dtElements = scriptDoc.getElementsByTag("record");


            for (Element recordElement : dtElements) {
                Elements elementsA = recordElement.getElementsByTag("a");
                for(Element eA:elementsA) {
                    String href=eA.attr("href");
                    String title=eA.text();
                    String date="";
                    Elements elements1=eA.parent().select("span");
                    for (Element ee:elements1){
                        if(ee.html().indexOf("-")>0){
                            date=ee.html().replace("[","").replace("]","");
                            if(date.length()<10){
                                date=LocalDate.now().getYear()+"-"+date;
                            }
                        }
                        //System.out.println(ee.html());
                    }
                   // String date=eA.parent().select("span:nth-last-child(1)").html();
                    if(href.indexOf("http")<0){
                        href=baseUri+href;
                    }
                    if(href.equals(lastUrl)){
                        //遇到相同的停止抓取
                        return policyNewsInfos;
                    }
                    //System.out.println(date);
                    PolicyNewsInfo policyNewsInfo=PolicyNewsInfo.builder()
                            .newsFrom(config.getNewsDept()).fetchUrl(href).fetchStatus(1)
                            .newsTitle(title)
                            .newsLevel(config.getNewsLevel())
                            .siteId(config.getId())
                            .newsType(config.getNewsTypeName())
                            .createTime(Timestamp.valueOf(LocalDate.from(formatter.parse(date)).atTime(LocalTime.MIDNIGHT))).build();
//                    if(href.indexOf("http")==-1) {
//                        href = URL_FIRST + href;
//                    }#\37 382343 > div > li:nth-child(1) > span.bt_time
                   // System.out.println(href+"::"+title+"::"+date);
                    try {
                        this.fetchContent(href,config.getContentSelector(),policyNewsInfo);
                    } catch (Exception ex) {
                        ex.printStackTrace();
                    }
                    policyNewsInfos.add(policyNewsInfo);
                   // return policyNewsInfos;
                   // return;
                    //新增请求
                   // page.addTargetRequest(href);
                }
            }
        }
        return policyNewsInfos;
    }

    @Override
    public void fetchContent(String contentUrl,String cssSelector,PolicyNewsInfo newsInfo) throws Exception {

        Document response = Jsoup.connect(contentUrl)
                .ignoreContentType(true)
                .method(Connection.Method.GET)
                // .proxy("127.0.0.1",8866)
                // .header("Content-Type","application/json;charset=UTF-8")
                //.header("Content-Type","application/json;charset=UTF-8")
                // .header("sec-ch-ua-platform","Windows")
                .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.82 Safari/537.36")
                //.cookies(cookie)
                .execute().parse();
        //System.out.println(response.body());
        Elements bodys =response.select(cssSelector);
       // String bodyStr=bodys.html();
        Element removeItem=null;
        Elements hrefs = bodys.get(0).getElementsByTag("a");
        for(Element e:hrefs){
            if(StringUtils.isNotEmpty(e.attr("href"))&&e.attr("href").indexOf("http")==-1) {
                e.attr("href", baseUri + e.attr("href"));
            }
        }
        Elements imgs = bodys.get(0).getElementsByTag("img");
        for(Element e:imgs){
            if(StringUtils.isNotEmpty(e.attr("src"))&&e.attr("src").indexOf("http")==-1){
                e.attr("src", baseUri +e.attr("src"));
            }
        }
        for(Element e:bodys.get(0).children()){
            if(e.hasClass("bt_xx")){
               // bodyStr=bodyStr.replace(e.html(),"");
                //System.out.println("new={}"+bodyStr);
                //System.out.println("new={}"+e.html());
               removeItem=e;
            }
            if(e.hasClass("gn")){
                removeItem=e;
            }
        }
        if(!ObjectUtils.isEmpty(removeItem)){
            bodys.remove(removeItem);
        }
       System.out.println(bodys.html());
        newsInfo.setNewsContent(bodys.html());

    }

    private void initBaseUrl(String urlStr) throws MalformedURLException {
        URL url = new URL(urlStr);
        baseUri="http://"+url.getHost();
        System.out.println(baseUri);
    }

    public static void main(String[] args){
        GovNewsProcessor processor=new GovNewsProcessor();
        String siteUrl="http://rsj.ningbo.gov.cn/col/col1229114053/index.html";
        String pageUrl="http://rsj.ningbo.gov.cn/art/2022/4/22/art_1229114053_58929039.html";
        try {
            processor.initBaseUrl(siteUrl);
            processor.fetchContent(pageUrl,"#ivs_content",new PolicyNewsInfo());
//            List<PolicyNewsInfo> list= processor.fetchPage(PolicySiteConfig.builder().siteUrl(siteUrl).contentSelector("#ivs_content").build(),null);
//            list.forEach(e->{
//                System.out.println(e.getNewsTitle()+"::"+e.getFetchUrl()+"::"+e.getNewsContent());
//            });
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
